package com.dianping.ri;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.Version;

import com.dianping.entity.DocInfo;
public class ReverseIndex {
	/**
	 *文档路径 
	 */
	private static final  String docPath="D:\\Users\\sky\\RelNotes";
	/**
	 *倒排结构 
	 */
	private static Map<String,ArrayList<DocInfo>> riMap = new HashMap<String, ArrayList<DocInfo>>();

	public static void main(String[] args){
		File docFile = new File(docPath);
		LinkedList<File> dirQueue = new LinkedList<File>();
		List<File> fileList = new ArrayList<File>();
		if(docFile.isDirectory()){
			dirQueue.offer(docFile);
		}else{
			fileList.add(docFile);
		}
		
		File queueFile = dirQueue.poll();
		while(queueFile!=null){
			for(File qFile:queueFile.listFiles()){
				if(qFile.isDirectory()){
					dirQueue.offer(qFile);
				}else{
					fileList.add(qFile);
				}
			}
			queueFile=dirQueue.poll();
		}
		int docSum = fileList.size();
		for(File file:fileList){
			
			DocInfo docInfo = new DocInfo("",0);
			String text = loadFileToString(file);
			
			StringReader sr = new StringReader(text);
			Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_35);
			TokenStream ts = analyzer.tokenStream("", sr);
			ts.addAttribute(TermAttribute.class);
			try {
				while(ts.incrementToken()){
				    TermAttribute ta = ts.getAttribute(TermAttribute.class); 
				    String taTerm = ta.term();
				    ArrayList<DocInfo> docInfoListRep = new ArrayList<DocInfo>();
				    if(riMap.containsKey(taTerm)){
				    	ArrayList<DocInfo> docInfoList = riMap.get(taTerm);
				    	
				    	docInfoListRep.addAll(docInfoList);
				    	for(DocInfo docInfo1:docInfoListRep){
				    		if(docInfo1.getDocName().equals(file.getName())){
				    			//termFreq+1
				    			docInfo1.setTermFreq(docInfo1.getTermFreq()+1);
				    			riMap.put(taTerm, docInfoListRep);
				    		}else{
							    docInfo.setDocName(file.getName());
							    docInfo.setTermFreq(1);
							    docInfoList.add(docInfo);
							    //riMap.put(taTerm, docInfoList);
				    		}
				    	}
				    	docInfoList=null;
				    	docInfoListRep=null;
				    }else{
				    	ArrayList<DocInfo> docInfoList = new ArrayList<DocInfo>();
					    docInfo.setDocName(file.getName());
					    docInfo.setTermFreq(1);
					    docInfoList.add(docInfo);
					    riMap.put(taTerm, docInfoList);
					    docInfoList=null;
				    }
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		computeDocFreq(riMap,docSum);
	}
	public static String loadFileToString(File file){
		//Map<String,List<DocInfo>> map = new HashMap<String, List<DocInfo>>();
		try {
			System.out.println(file.getCanonicalPath());
			FileReader fileReader = new FileReader(file);
			BufferedReader br = new BufferedReader(fileReader);
			StringBuilder sb = new StringBuilder();
			
			String lineString = br.readLine();
			while(lineString!=null){
				sb.append(lineString);
				lineString=br.readLine();
			}
			return sb.toString();
		} catch (IOException e) {
			e.printStackTrace();
			return "";
		}
	}
	public static void computeDocFreq(Map<String, ArrayList<DocInfo>> riMap,int docSum){
		//计算docFreq
		Iterator<String> riMapIt = riMap.keySet().iterator();
		Map<String,Integer> docFreqMap = new HashMap<String, Integer>();
		Map<String,Double> idfMap = new HashMap<String, Double>();
		while(riMapIt.hasNext()){
			String mapTerm = (String) riMapIt.next();
			docFreqMap.put(mapTerm, (Integer) riMap.get(mapTerm).size());
		}
		Iterator<String> docFreqMapIt = docFreqMap.keySet().iterator();
		while(docFreqMapIt.hasNext()){
			String idfTerm = docFreqMapIt.next();
			double tmp = docSum / docFreqMap.get(idfTerm);
			double idf = Math.log(tmp)/Math.log(2);
			idfMap.put(idfTerm, idf);
		}
		String outPath = "D:\\term.txt";
		try {
			Iterator<String> idfIt = idfMap.keySet().iterator();
			FileOutputStream fos = new FileOutputStream(outPath);
			while(idfIt.hasNext()){
				String docTerm=(String) idfIt.next();
				double docFreq = idfMap.get(docTerm);
				String unitTerm = docTerm+"-----"+docFreq;
				fos.write(unitTerm.getBytes());
				fos.write("\n".getBytes());
				//String docTerm=(String) docFreqMapIt.next();
				//int docFreq = docFreqMap.get(docTerm);
				//System.out.println(docFreq);
			}
			
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}
